#pragma once
#include <cppjieba/Jieba.hpp>
#include <utfcpp/utf8.h>
#include <string>
#include <set>
#include <map>

class KeyWordProcessor {
public:
    KeyWordProcessor();
	
    void setCnStopWords(const std::string& path);
    void setEnStopWords(const std::string& path);

    void getEnWordFrequency(const std::vector<std::string> &corpus); 
    void getCnWordFrequency(const std::vector<std::string> &corpus); 

    // chDir: 中文语料库
    // enDir: 英文语料库
    void process(const std::string& chDir, const std::string& enDir);
   
private:
    void create_cn_dict(const std::string& path);
    void build_cn_index(const std::string& dict, const std::string& index);

    void create_en_dict(const std::string& path);
    void build_en_index(const std::string& dict, const std::string& index);
private:
    cppjieba::Jieba m_tokenizer;
    std::set<std::string> m_enStopWords;
    std::set<std::string> m_cnStopWords;
    std::map<std::string, int> m_cnWordFrequency;
    std::map<std::string, int> m_enWordFrequency;
    std::map<char, std::set<int>> m_enAlphaIndex;
    std::map<std::string, std::set<int>> m_cnAlphaIndex;
};